{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>age</th>\n",
       "      <th>Private</th>\n",
       "      <th>Self-employed-incorporated</th>\n",
       "      <th>State government</th>\n",
       "      <th>Self-employed-not incorporated</th>\n",
       "      <th>Not in universe</th>\n",
       "      <th>Without pay</th>\n",
       "      <th>Federal government</th>\n",
       "      <th>Never worked</th>\n",
       "      <th>...</th>\n",
       "      <th>1.2</th>\n",
       "      <th>Not in universe.12</th>\n",
       "      <th>Yes.3</th>\n",
       "      <th>No.3</th>\n",
       "      <th>2.3</th>\n",
       "      <th>0.3</th>\n",
       "      <th>1.3</th>\n",
       "      <th>weeks worked in year</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 511 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  age   Private   Self-employed-incorporated   State government  \\\n",
       "0   0   33         1                            0                  0   \n",
       "1   1   63         1                            0                  0   \n",
       "2   2   71         0                            0                  0   \n",
       "3   3   43         0                            0                  0   \n",
       "4   4   57         0                            0                  0   \n",
       "\n",
       "    Self-employed-not incorporated   Not in universe   Without pay  \\\n",
       "0                                0                 0             0   \n",
       "1                                0                 0             0   \n",
       "2                                0                 1             0   \n",
       "3                                0                 0             0   \n",
       "4                                0                 0             0   \n",
       "\n",
       "    Federal government   Never worked  ...   1.2   Not in universe.12   Yes.3  \\\n",
       "0                    0              0  ...     0                    1       0   \n",
       "1                    0              0  ...     0                    1       0   \n",
       "2                    0              0  ...     0                    1       0   \n",
       "3                    0              0  ...     0                    1       0   \n",
       "4                    0              0  ...     0                    1       0   \n",
       "\n",
       "    No.3   2.3   0.3   1.3  weeks worked in year   94   95  \n",
       "0      0     1     0     0                    52    0    1  \n",
       "1      0     1     0     0                    52    0    1  \n",
       "2      0     1     0     0                     0    0    1  \n",
       "3      0     1     0     0                    52    0    1  \n",
       "4      0     1     0     0                    52    0    1  \n",
       "\n",
       "[5 rows x 511 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_path = '../data/X_train'\n",
    "y_train_path = '../data/Y_train'\n",
    "x_test_path  = '../data/X_test'\n",
    "\n",
    "x_train = pd.read_csv(x_train_path)\n",
    "y_train = pd.read_csv(y_train_path)\n",
    "x_test  = pd.read_csv(x_test_path)\n",
    "\n",
    "x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train :\n",
      " [[-0.42755297  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [ 1.19978057  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [ 1.63373618 -1.00040557 -0.18224011 ... -1.45536172 -1.01485524\n",
      "   1.01485524]\n",
      " ...\n",
      " [-1.34970865 -1.00040557 -0.18224011 ... -1.10738917  0.98536221\n",
      "  -0.98536221]\n",
      " [ 0.3861138   0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [ 0.3861138  -1.00040557 -0.18224011 ... -1.45536172 -1.01485524\n",
      "   1.01485524]] (54256, 510) \n",
      "\n",
      "y_train :\n",
      " [1 0 0 ... 0 0 0] (54256,) \n",
      "\n",
      "x_test :\n",
      " [[-0.21057517  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [ 0.3861138   0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [ 1.47100282 -1.00040557 -0.18224011 ... -1.45536172  0.98536221\n",
      "  -0.98536221]\n",
      " ...\n",
      " [-0.15633072  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [-1.29546419 -1.00040557 -0.18224011 ...  0.28450104  0.98536221\n",
      "  -0.98536221]\n",
      " [-1.02424194 -1.00040557 -0.18224011 ... -0.36794749  0.98536221\n",
      "  -0.98536221]] (27622, 510)\n"
     ]
    }
   ],
   "source": [
    "x_train = np.array(x_train)[:, 1:]\n",
    "y_train = (np.array(y_train)[:, 1:]).flatten()\n",
    "x_test  = np.array(x_test)[:, 1:]\n",
    "\n",
    "# normalize\n",
    "scaler = StandardScaler().fit(x_train)\n",
    "# print(scaler.mean, scaler.std)\n",
    "x_train = scaler.transform(x_train)\n",
    "x_test  = scaler.transform(x_test)\n",
    "\n",
    "print('x_train :\\n',x_train,x_train.shape,'\\n')\n",
    "print('y_train :\\n',y_train,y_train.shape,'\\n')\n",
    "print('x_test :\\n',x_test,x_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_training_set :  (48830, 510) \n",
      " [[ 0.6030916  -1.00040557 -0.18224011 ...  0.80645987  0.98536221\n",
      "  -0.98536221]\n",
      " [ 0.6030916   0.9995946  -0.18224011 ...  0.80645987  0.98536221\n",
      "  -0.98536221]\n",
      " [-0.80726413 -1.00040557 -0.18224011 ... -0.75941661 -1.01485524\n",
      "   1.01485524]\n",
      " ...\n",
      " [ 0.44035825  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [ 1.52524727  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [-0.80726413 -1.00040557 -0.18224011 ... -1.45536172 -1.01485524\n",
      "   1.01485524]]\n",
      "------------------------------------------------------------------------\n",
      "y_training_set :  (48830,) \n",
      " [0 1 0 ... 0 1 0]\n",
      "------------------------------------------------------------------------\n",
      "x_validation_set :  (5426, 510) \n",
      " [[-0.80726413  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [ 0.00640264  0.9995946  -0.18224011 ... -0.32445093  0.98536221\n",
      "  -0.98536221]\n",
      " [-0.64453078  0.9995946  -0.18224011 ...  0.45848731  0.98536221\n",
      "  -0.98536221]\n",
      " ...\n",
      " [ 1.03704721  0.9995946  -0.18224011 ...  0.80645987  0.98536221\n",
      "  -0.98536221]\n",
      " [-0.10208627  0.9995946  -0.18224011 ...  0.80645987 -1.01485524\n",
      "   1.01485524]\n",
      " [-0.26481962  0.9995946  -0.18224011 ...  0.80645987  0.98536221\n",
      "  -0.98536221]]\n",
      "------------------------------------------------------------------------\n",
      "y_validation_set :  (5426,) \n",
      " [0 0 0 ... 0 1 1]\n"
     ]
    }
   ],
   "source": [
    "# 切分validation set\n",
    "x_training_set, x_validation_set, y_training_set, y_validation_set = train_test_split(x_train, y_train, test_size = 0.1)\n",
    "\n",
    "print('x_training_set : ', x_training_set.shape, '\\n', x_training_set)\n",
    "print('------------------------------------------------------------------------')\n",
    "print('y_training_set : ', y_training_set.shape, '\\n', y_training_set)\n",
    "print('------------------------------------------------------------------------')\n",
    "print('x_validation_set : ', x_validation_set.shape, '\\n', x_validation_set)\n",
    "print('------------------------------------------------------------------------')\n",
    "print('y_validation_set : ', y_validation_set.shape, '\\n', y_validation_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score_training_set:  0.8853573622772886 \t score_validation_set:  0.881865093991891\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gehao/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score_training_set:  0.8857259881220562 \t score_validation_set:  0.8822336896424622\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gehao/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score_training_set:  0.8858283841900471 \t score_validation_set:  0.8824179874677479\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gehao/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score_training_set:  0.8858283841900471 \t score_validation_set:  0.8826022852930335\n",
      "score_training_set:  0.8858283841900471 \t score_validation_set:  0.8826022852930335\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gehao/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    }
   ],
   "source": [
    "best_score = 0.0\n",
    "for C in [0.01, 0.1, 1, 10, 100]:\n",
    "    lr_clf = LogisticRegression(C = C)\n",
    "    lr_clf.fit(x_training_set, y_training_set)\n",
    "    score_training = lr_clf.score(x_training_set, y_training_set)\n",
    "    score_validation = lr_clf.score(x_validation_set, y_validation_set)\n",
    "    print('score_training_set: ', score_training, '\\t', 'score_validation_set: ', score_validation)\n",
    "    if score_validation > best_score:\n",
    "        best_score = score_validation\n",
    "        best_parameters = C"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score_train:  0.8857269242111472\n",
      "y_test_predict:\n",
      " [0 0 0 ... 1 0 0]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gehao/anaconda3/envs/ml/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    }
   ],
   "source": [
    "lr = LogisticRegression(C = best_parameters)\n",
    "lr.fit(x_train, y_train)\n",
    "score = lr.score(x_train, y_train)\n",
    "print('score_train: ',score)\n",
    "y_test_predict = lr.predict(x_test)\n",
    "print('y_test_predict:\\n',y_test_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.8 64-bit ('ml': conda)",
   "language": "python",
   "name": "python36864bitmlconda4727b916de1c4a8f8265036fed6d2bb8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
